{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#通过输入，神经网络的参数来进行输出\n",
    "def inference(input_tensor,avger,weight1,biase1,weight2,biase2):\n",
    "    #判断是否使用滑动平均模型\n",
    "    #不使用滑动平均模型\n",
    "    if avger == None:\n",
    "        #计算第一层神经网络的前向传播，并使用RELU激活函数\n",
    "        layer1 = tf.nn.relu(tf.matmul(input_tensor,weight1) + biase1)\n",
    "        #计算第二层的神经网络的前向传播\n",
    "        return tf.matmul(layer1,weight2) + biase2\n",
    "    #使用滑动平均模型\n",
    "    else:\n",
    "        #通过滑动平均模型更新参数,利用更新后的参数进行前向传播计算\n",
    "        layer1 = tf.nn.relu(tf.matmul(input_tensor,avger.average(weight1))+avger.average(biase1))\n",
    "        #计算第二层神经网络的前向传播\n",
    "        return tf.matmul(layer1,avger.average(weight2)) + avger.average(biase2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-3-6a5a234537cb>:58: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "#定义第一层网络的输入节点数\n",
    "#输入为784是因为，每张手写数字的图片大小为28*28的矩阵，将它转成一维向量就是1*784\n",
    "input_node = 784\n",
    "#定义最后一层网络的输出节点数\n",
    "#输出为一个1*10的向量，正好可以表示0-9,10个数字\n",
    "output_node = 10\n",
    "\n",
    "#定义神经网络的结构\n",
    "#定义第一层隐藏层的节点数\n",
    "layer1_node = 500\n",
    "#定义每次训练图片的张数，可以有效防止内存溢出\n",
    "batch_size = 100\n",
    "#定义基础的学习率用于指数衰减的学习率中\n",
    "learning_rate_base = 0.8\n",
    "#设置学习率的衰减率\n",
    "learning_rate_decay = 0.99\n",
    "#设置L1正则化中，模型复杂度在损失函数中所占的比例\n",
    "regularization_rate = 0.0001\n",
    "#设置训练的轮数\n",
    "training_steps = 30000\n",
    "#设置滑动平均的衰减率\n",
    "moving_average_decay = 0.99\n",
    "#定义输入节点\n",
    "x = tf.placeholder(tf.float32,[None,input_node],name=\"x_input\")\n",
    "#定义输出节点\n",
    "y_ = tf.placeholder(tf.float32,[None,output_node],name=\"y_output\")\n",
    "#设置第一层神经网络的权重\n",
    "weight1 = tf.Variable(tf.truncated_normal([input_node,layer1_node],stddev=0.1))\n",
    "#设置第一层神经网络的偏置\n",
    "biase1 = tf.Variable(tf.constant(0.1,shape=[layer1_node]))\n",
    "#设置第二层神经网络的权重\n",
    "weight2 = tf.Variable(tf.truncated_normal([layer1_node,output_node],stddev=0.1))\n",
    "#设置第二层神经网络的偏置\n",
    "biase2 = tf.Variable(tf.constant(0.1,shape=[output_node]))\n",
    "#计算神经网络的前向传播结果\n",
    "#不适用参数的滑动平均值\n",
    "y = inference(x,None,weight1,biase1,weight2,biase2)\n",
    "#定义滑动平均的global_step\n",
    "global_step = tf.Variable(0,trainable=False)\n",
    "#初始化滑动平均\n",
    "averge = tf.train.ExponentialMovingAverage(moving_average_decay,global_step)\n",
    "#定义滑动平均所更新的列表，tf.trainable_variables()获取所有的列表\n",
    "variables_averages = averge.apply(tf.trainable_variables())\n",
    "#通过滑动平均，获取神经网络的前向传播参数\n",
    "# averge_y = inference(x,averge,weight1,biase1,weight2,biase2)\n",
    "#使用交叉熵作为损失函数\n",
    "# cross_entropy = -tf.reduce_sum(y_*tf.log(y))\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=tf.argmax(y_,1),logits=y)\n",
    "#计算交叉熵的平均值\n",
    "cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
    "#计算L2正则化损失函数\n",
    "regularizer = tf.contrib.layers.l2_regularizer(regularization_rate)\n",
    "#计算模型参数的L2正则化\n",
    "regularization = regularizer(weight1) + regularizer(weight2)\n",
    "#计算中的损失\n",
    "loss = cross_entropy_mean + regularization\n",
    "# 下载minist的手写数字的数据集\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "#设置指数衰减学习率\n",
    "learning_rate = tf.train.exponential_decay(learning_rate_base,\n",
    "                                           global_step,mnist.train.num_examples/batch_size\n",
    "                                           ,learning_rate_decay)\n",
    "#随机梯度下降优化损失函数\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "#每迭代一次需要更新神经网络中的参数\n",
    "train_op = tf.group(train_step)#, variables_averages\n",
    "# correct_prediction = tf.equal(tf.argmax(averge_y,1),tf.argmax(y_,1))\n",
    "correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))\n",
    "#计算数据的准确率\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:118: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "After 0 training step(s),validate accuracy using averagemodel is 0.23\n",
      "After 1000 training step(s),validate accuracy using averagemodel is 0.9744\n",
      "After 2000 training step(s),validate accuracy using averagemodel is 0.9778\n",
      "After 3000 training step(s),validate accuracy using averagemodel is 0.976\n",
      "After 4000 training step(s),validate accuracy using averagemodel is 0.981\n",
      "After 5000 training step(s),validate accuracy using averagemodel is 0.9824\n",
      "After 6000 training step(s),validate accuracy using averagemodel is 0.9826\n",
      "After 7000 training step(s),validate accuracy using averagemodel is 0.9832\n",
      "After 8000 training step(s),validate accuracy using averagemodel is 0.9844\n",
      "After 9000 training step(s),validate accuracy using averagemodel is 0.984\n",
      "After 10000 training step(s),validate accuracy using averagemodel is 0.9842\n",
      "After 11000 training step(s),validate accuracy using averagemodel is 0.9842\n",
      "After 12000 training step(s),validate accuracy using averagemodel is 0.9842\n",
      "After 13000 training step(s),validate accuracy using averagemodel is 0.9824\n",
      "After 14000 training step(s),validate accuracy using averagemodel is 0.9852\n",
      "After 15000 training step(s),validate accuracy using averagemodel is 0.9846\n",
      "After 16000 training step(s),validate accuracy using averagemodel is 0.9832\n",
      "After 17000 training step(s),validate accuracy using averagemodel is 0.9832\n",
      "After 18000 training step(s),validate accuracy using averagemodel is 0.9836\n",
      "After 19000 training step(s),validate accuracy using averagemodel is 0.9834\n",
      "After 20000 training step(s),validate accuracy using averagemodel is 0.9856\n",
      "After 21000 training step(s),validate accuracy using averagemodel is 0.9856\n",
      "After 22000 training step(s),validate accuracy using averagemodel is 0.9854\n",
      "After 23000 training step(s),validate accuracy using averagemodel is 0.9852\n",
      "After 24000 training step(s),validate accuracy using averagemodel is 0.9836\n",
      "After 25000 training step(s),validate accuracy using averagemodel is 0.9836\n",
      "After 26000 training step(s),validate accuracy using averagemodel is 0.9854\n",
      "After 27000 training step(s),validate accuracy using averagemodel is 0.9856\n",
      "After 28000 training step(s),validate accuracy using averagemodel is 0.9842\n",
      "After 29000 training step(s),validate accuracy using averagemodel is 0.9854\n",
      "After 30000 training step(s),test accuracy using averagemodel is 0.9836\n"
     ]
    }
   ],
   "source": [
    "#初始化会话\n",
    "with tf.Session() as sess:\n",
    "    #初始化参数\n",
    "    tf.initialize_all_variables().run()\n",
    "    # 验证数据\n",
    "    validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}\n",
    "    #迭代训练\n",
    "    for i in range(training_steps):\n",
    "        batch_x,batch_y = mnist.train.next_batch(batch_size)\n",
    "        sess.run(train_op,feed_dict={x:batch_x,y_:batch_y})\n",
    "        if i % 1000 == 0:\n",
    "            validate_acc = sess.run(accuracy,feed_dict=validate_feed)\n",
    "            print(\"After %d training step(s),validate accuracy using average\"\n",
    "                  \"model is %g\" % (i, validate_acc))\n",
    "            validate_acc = sess.run(accuracy,feed_dict={x: mnist.train.images, y_: mnist.train.labels})\n",
    "            print(\"After %d training step(s),train accuracy using average\"\n",
    "                  \"model is %g\" % (i, validate_acc))\n",
    "            validate_acc = sess.run(accuracy,feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "            print(\"After %d training step(s),test accuracy using average\"\n",
    "                  \"model is %g\" % (i, validate_acc))\n",
    "\n",
    "    # 测试数据\n",
    "#     test_feed = {x: mnist.test.images, y_: mnist.test.labels}\n",
    "    #训练完成之后，测试测试集的准确率\n",
    "    test_acc = sess.run(accuracy,feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "    print(\"After %d training step(s),test accuracy using average\"\n",
    "          \"model is %g\"%(training_steps,test_acc))\n",
    "    #0.9841\n",
    "    #不使用滑动平均模型的准确率\n",
    "    #0.9838\n",
    "    #不使用L2正则化\n",
    "    #0.9845\n",
    "    #不使用指数衰减\n",
    "    #0.9681"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
